# encoding:utf-8
# user: ares at 18-9-5
# 爬取房天下深圳地区租房信息

import requests
from bs4 import BeautifulSoup
from pymongo import MongoClient
import time
import spderPublic

def getOnePageData(url,zfList):
    header={
        'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36'
    }
    wbdata=requests.get(url,headers=header).text
    datas=BeautifulSoup(wbdata,'html.parser')
    #print(datas)

    divs=datas.find_all("dd",attrs={'class':'info rel'})
    #data=datas.select('div[class="houseList"')
    #print(divs)

    for div in divs:
        ps=div.find_all('p')
        #print(ps)
        try:
            #subDiv=div.find_all('div')
            #for sd in subDiv:
            #    text=sd.text.strip()
            #    print(text)
            #print('========')


            # for index,p in enumerate(ps):
            #     text=p.text.strip()
            #     print(text)
            # print('=======================')

            roomMsg=ps[1].text.split('|')

            zutype = roomMsg[0].strip()                                             #租赁类型
            huxing = roomMsg[1].strip()                                             #户型
            area= roomMsg[2].strip()
            area=int(float(area[:len(area)-2]))                 #面积
            chaoxiang = ''                                          #朝向

            title=ps[0].text.strip()                                    #标题

            address=ps[2].text.strip()                                  #地址
            quyu = address.split('-')
            diqu=spderPublic.updateDist(quyu[0])                                                #地区
            jiedao=quyu[1]                                              #街道
            xiaoqu=quyu[2]                                              #小区

            price=int(div.find('span',attrs={'class':'price'}).text)
            #print(price)
            #price=int(float(ps[6].text.strip()[:len(ps[6].text.strip())-3]))        #价格
            jtms=div.find_all('div')[0].text.strip()

            #jtms = ''#ps[3].text.strip()  # 交通描述

            zfdict = {
                'title': title,
                'area': area,
                'diqu': diqu,
                'jiedao': jiedao,
                'xiaoqu': xiaoqu,
                'price': price,
                'jtms': jtms,
                'huxing': huxing,
                'chaoxiang': chaoxiang,
                'zutype': zutype,
                'laiyuan': '房天下'
            }

            if zfdict not in zfList:  # 去重复处理
                zfList.append(zfdict)

        except:
            print('\n'+'出错：'+ps[0].text.strip())
            continue

    return zfList

def insertData(datalist):
    #connect
    client=MongoClient('localhost:27017',connect=True)
    db=client['test']
    collection=db['zfdb']
    collection.insert(datalist)

if __name__ == '__main__':

    datalist=[]
    start=time.perf_counter()
    print('开始爬取房天下租房数据'.center(100//2,'-'))
    for i in range(1,101):
        url='http://sz.zu.fang.com/house/i3'+str(i)+'/'

        #print(url)
        datalist=getOnePageData(url,datalist)

        a='#'*i
        b='-'*(100-i)
        c=i
        dur=time.perf_counter()-start
        print('\r{:^3.0f}%[{}->{}]{:.2f}s'.format(c,a,b,dur),end='')

    insertData(datalist)

    print('\n'+'爬取房天下租房数据结束'.center(100 // 2,'-'))